{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From D:\\Miniconda\\Miniconda1\\lib\\site-packages\\tensorflow\\python\\util\\tf_should_use.py:170: initialize_all_variables (from tensorflow.python.ops.variables) is deprecated and will be removed after 2017-03-02.\n",
      "Instructions for updating:\n",
      "Use `tf.global_variables_initializer` instead.\n",
      "5000\n",
      "Loss: 9.99946\n",
      "Epsilon: 0.951717130256\n",
      "10000\n",
      "Loss: 7.03131\n",
      "Epsilon: 0.905789043856\n",
      "15000\n",
      "Loss: 0.0135831\n",
      "Epsilon: 0.862100896661\n",
      "20000\n",
      "Loss: 0.00143269\n",
      "Epsilon: 0.820543445547\n",
      "25000\n",
      "Loss: 0.000224503\n",
      "Epsilon: 0.781012775241\n",
      "30000\n",
      "Loss: 0.000427819\n",
      "Epsilon: 0.743410038475\n",
      "35000\n",
      "Loss: 1.38816e-05\n",
      "Epsilon: 0.707641208822\n",
      "40000\n",
      "Loss: 5.90028e-06\n",
      "Epsilon: 0.673616845575\n",
      "45000\n",
      "Loss: 1.65865e-06\n",
      "Epsilon: 0.641251870106\n",
      "50000\n",
      "Loss: 1.1249e-06\n",
      "Epsilon: 0.610465353116\n",
      "55000\n",
      "Loss: 1.71274e-06\n",
      "Epsilon: 0.581180312277\n",
      "60000\n",
      "Loss: 0.000101946\n",
      "Epsilon: 0.553323519733\n",
      "65000\n",
      "Loss: 1.03171e-05\n",
      "Epsilon: 0.526825318993\n",
      "70000\n",
      "Loss: 3.37203e-07\n",
      "Epsilon: 0.501619450753\n",
      "75000\n",
      "Loss: 1.40522e-06\n",
      "Epsilon: 0.477642887214\n",
      "80000\n",
      "Loss: 5.96705e-06\n",
      "Epsilon: 0.454835674476\n",
      "85000\n",
      "Loss: 6.84994e-07\n",
      "Epsilon: 0.433140782629\n",
      "90000\n",
      "Loss: 2.58871e-06\n",
      "Epsilon: 0.412503963143\n",
      "95000\n",
      "Loss: 1.86714e-06\n",
      "Epsilon: 0.39287361322\n",
      "100000\n",
      "Loss: 1.24457e-07\n",
      "Epsilon: 0.37420064676\n",
      "105000\n",
      "Loss: 2.68128e-08\n",
      "Epsilon: 0.35643837162\n",
      "110000\n",
      "Loss: 6.58361e-08\n",
      "Epsilon: 0.339542372861\n",
      "115000\n",
      "Loss: 3.79655e-07\n",
      "Epsilon: 0.323470401685\n",
      "115000\n",
      "Loss: 1.2246e-07\n",
      "Epsilon: 0.323470401685\n",
      "120000\n",
      "Loss: 2.98495e-07\n",
      "Epsilon: 0.308182269793\n",
      "125000\n",
      "Loss: 2.35762e-07\n",
      "Epsilon: 0.293639748892\n",
      "130000\n",
      "Loss: 9.93927e-08\n",
      "Epsilon: 0.279806475104\n",
      "135000\n",
      "Loss: 3.06277e-07\n",
      "Epsilon: 0.266647858039\n",
      "140000\n",
      "Loss: 3.03686e-08\n",
      "Epsilon: 0.254130994302\n",
      "145000\n",
      "Loss: 7.95459e-08\n",
      "Epsilon: 0.242224585213\n",
      "150000\n",
      "Loss: 1.49012e-09\n",
      "Epsilon: 0.230898858547\n",
      "155000\n",
      "Loss: 4.16216e-08\n",
      "Epsilon: 0.220125494088\n",
      "160000\n",
      "Loss: 1.56987e-07\n",
      "Epsilon: 0.209877552815\n",
      "165000\n",
      "Loss: 1.10838e-08\n",
      "Epsilon: 0.200129409535\n",
      "170000\n",
      "Loss: 1.52884e-08\n",
      "Epsilon: 0.190856688812\n",
      "175000\n",
      "Loss: 1.13419e-08\n",
      "Epsilon: 0.182036204016\n",
      "180000\n",
      "Loss: 1.6884e-07\n",
      "Epsilon: 0.173645899339\n",
      "185000\n",
      "Loss: 2.39597e-09\n",
      "Epsilon: 0.16566479465\n",
      "190000\n",
      "Loss: 5.67525e-10\n",
      "Epsilon: 0.15807293303\n",
      "195000\n",
      "Loss: 2.59715e-08\n",
      "Epsilon: 0.150851330871\n",
      "200000\n",
      "Loss: 1.8219e-09\n",
      "Epsilon: 0.143981930404\n",
      "200000\n",
      "Loss: 1.87501e-09\n",
      "Epsilon: 0.143981930404\n",
      "205000\n",
      "Loss: 4.67026e-09\n",
      "Epsilon: 0.137447554552\n",
      "205000\n",
      "Loss: 1.36442e-09\n",
      "Epsilon: 0.137447554552\n",
      "210000\n",
      "Loss: 2.10704e-08\n",
      "Epsilon: 0.13123186397\n",
      "215000\n",
      "Loss: 2.95675e-08\n",
      "Epsilon: 0.125319316196\n",
      "220000\n",
      "Loss: 6.31007e-10\n",
      "Epsilon: 0.119695126779\n",
      "225000\n",
      "Loss: 1.84082e-10\n",
      "Epsilon: 0.114345232316\n",
      "230000\n",
      "Loss: 1.06498e-08\n",
      "Epsilon: 0.109256255286\n",
      "235000\n",
      "Loss: 1.56069e-10\n",
      "Epsilon: 0.104415470593\n",
      "240000\n",
      "Loss: 1.22618e-09\n",
      "Epsilon: 0.0998107737565\n",
      "245000\n",
      "Loss: 1.04431e-07\n",
      "Epsilon: 0.0954306506344\n",
      "245000\n",
      "Loss: 2.22397e-08\n",
      "Epsilon: 0.0954306506344\n",
      "250000\n",
      "Loss: 4.05998e-10\n",
      "Epsilon: 0.0912641486377\n",
      "250000\n",
      "Loss: 9.26229e-10\n",
      "Epsilon: 0.0912641486377\n",
      "255000\n",
      "Loss: 2.09366e-09\n",
      "Epsilon: 0.0873008493411\n",
      "260000\n",
      "Loss: 4.03998e-10\n",
      "Epsilon: 0.0835308424322\n",
      "265000\n",
      "Loss: 1.10451e-08\n",
      "Epsilon: 0.0799447009298\n",
      "270000\n",
      "Loss: 1.28293e-09\n",
      "Epsilon: 0.0765334576124\n",
      "275000\n",
      "Loss: 1.96023e-08\n",
      "Epsilon: 0.0732885825946\n",
      "280000\n",
      "Loss: 1.67856e-09\n",
      "Epsilon: 0.070201961999\n",
      "285000\n",
      "Loss: 2.07092e-09\n",
      "Epsilon: 0.0672658776661\n",
      "290000\n",
      "Loss: 5.18958e-10\n",
      "Epsilon: 0.0644729878558\n",
      "295000\n",
      "Loss: 2.60825e-09\n",
      "Epsilon: 0.0618163088889\n",
      "300000\n",
      "Loss: 7.29051e-10\n",
      "Epsilon: 0.0592891976842\n",
      "305000\n",
      "Loss: 3.12139e-10\n",
      "Epsilon: 0.0568853351472\n",
      "305000\n",
      "Loss: 1.98215e-09\n",
      "Epsilon: 0.0568853351472\n",
      "310000\n",
      "Loss: 5.47152e-10\n",
      "Epsilon: 0.0545987103696\n",
      "315000\n",
      "Loss: 8.69079e-08\n",
      "Epsilon: 0.0524236055984\n",
      "320000\n",
      "Loss: 1.22454e-09\n",
      "Epsilon: 0.0503545819386\n",
      "325000\n",
      "Loss: 2.80306e-09\n",
      "Epsilon: 0.0483864657534\n",
      "330000\n",
      "Loss: 2.00271e-10\n",
      "Epsilon: 0.0465143357272\n",
      "335000\n",
      "Loss: 4.57658e-10\n",
      "Epsilon: 0.0447335105598\n",
      "340000\n",
      "Loss: 1.68571e-08\n",
      "Epsilon: 0.0430395372607\n",
      "345000\n",
      "Loss: 6.35919e-10\n",
      "Epsilon: 0.0414281800143\n",
      "345000\n",
      "Loss: 1.37775e-09\n",
      "Epsilon: 0.0414281800143\n",
      "350000\n",
      "Loss: 4.8185e-10\n",
      "Epsilon: 0.0398954095881\n",
      "350000\n",
      "Loss: 9.09495e-10\n",
      "Epsilon: 0.0398954095881\n",
      "355000\n",
      "Loss: 5.26234e-10\n",
      "Epsilon: 0.0384373932577\n",
      "360000\n",
      "Loss: 8.51287e-10\n",
      "Epsilon: 0.0370504852228\n",
      "365000\n",
      "Loss: 4.77976e-09\n",
      "Epsilon: 0.035731217491\n",
      "370000\n",
      "Loss: 3.25235e-10\n",
      "Epsilon: 0.0344762912056\n",
      "375000\n",
      "Loss: 4.60568e-10\n",
      "Epsilon: 0.0332825683974\n",
      "380000\n",
      "Loss: 3.24508e-10\n",
      "Epsilon: 0.0321470641376\n",
      "385000\n",
      "Loss: 2.10457e-10\n",
      "Epsilon: 0.031066939074\n",
      "390000\n",
      "Loss: 2.04673e-09\n",
      "Epsilon: 0.0300394923313\n",
      "395000\n",
      "Loss: 3.065e-10\n",
      "Epsilon: 0.0290621547576\n",
      "400000\n",
      "Loss: 1.3526e-09\n",
      "Epsilon: 0.0281324824998\n",
      "405000\n",
      "Loss: 3.17959e-10\n",
      "Epsilon: 0.0272481508931\n",
      "410000\n",
      "Loss: 4.30919e-10\n",
      "Epsilon: 0.0264069486477\n",
      "415000\n",
      "Loss: 2.54295e-10\n",
      "Epsilon: 0.02560677232\n",
      "415000\n",
      "Loss: 1.47884e-10\n",
      "Epsilon: 0.02560677232\n",
      "420000\n",
      "Loss: 2.663e-10\n",
      "Epsilon: 0.0248456210523\n",
      "425000\n",
      "Loss: 1.78989e-10\n",
      "Epsilon: 0.0241215915699\n",
      "430000\n",
      "Loss: 2.73758e-10\n",
      "Epsilon: 0.0234328734221\n",
      "435000\n",
      "Loss: 3.28509e-10\n",
      "Epsilon: 0.0227777444547\n",
      "440000\n",
      "Loss: 1.48793e-10\n",
      "Epsilon: 0.022154566504\n",
      "445000\n",
      "Loss: 1.30603e-10\n",
      "Epsilon: 0.0215617813007\n",
      "450000\n",
      "Loss: 8.47649e-11\n",
      "Epsilon: 0.0209979065729\n",
      "455000\n",
      "Loss: 2.79761e-10\n",
      "Epsilon: 0.02046153234\n",
      "460000\n",
      "Loss: 1.70985e-10\n",
      "Epsilon: 0.0199513173872\n",
      "465000\n",
      "Loss: 2.6339e-10\n",
      "Epsilon: 0.0194659859112\n",
      "470000\n",
      "Loss: 1.20328e-08\n",
      "Epsilon: 0.0190043243307\n",
      "475000\n",
      "Loss: 2.18279e-10\n",
      "Epsilon: 0.0185651782511\n",
      "480000\n",
      "Loss: 5.30599e-10\n",
      "Epsilon: 0.0181474495785\n",
      "485000\n",
      "Loss: 2.86673e-10\n",
      "Epsilon: 0.0177500937737\n",
      "490000\n",
      "Loss: 9.04038e-11\n",
      "Epsilon: 0.0173721172402\n",
      "495000\n",
      "Loss: 3.61979e-10\n",
      "Epsilon: 0.0170125748398\n",
      "500000\n",
      "Loss: 2.16278e-10\n",
      "Epsilon: 0.0166705675291\n",
      "500000\n",
      "Loss: 1.52795e-10\n",
      "Epsilon: 0.0166705675291\n",
      "****************** Agent is in the State 0 **********************\n",
      "Step is  1 ****************** Agent is in the State 1 **********************\n",
      "Step is  2 ****************** Agent is in the State 11 **********************\n",
      "Step is  3 ****************** Agent is in the State 12 **********************\n",
      "Step is  4 ****************** Agent is in the State 13 **********************\n",
      "Step is  5 ****************** Agent is in the State 14 **********************\n",
      "Step is  6 ****************** Agent is in the State 24 **********************\n",
      "Step is  7 ****************** Agent is in the State 25 **********************\n",
      "Step is  8 ****************** Agent is in the State 35 **********************\n",
      "Step is  9 ****************** Agent is in the State 36 **********************\n",
      "Step is  10 ****************** Agent is in the State 37 **********************\n",
      "Step is  11 ****************** Agent is in the State 38 **********************\n",
      "Step is  12 ****************** Agent is in the State 39 **********************\n",
      "Step is  13 ****************** Agent is in the State 49 **********************\n",
      "Step is  14 ****************** Agent is in the State 59 **********************\n",
      "Step is  15 ****************** Agent is in the State 69 **********************\n",
      "Step is  16 ****************** Agent is in the State 79 **********************\n",
      "Step is  17 ****************** Agent is in the State 89 **********************\n",
      "Step is  18 ****************** Agent is in the State 99 **********************\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmUVPWd9/H3t3qhQYSmMiiLOmYDjSSItFGzIIuOS9A4\nmaBg3CI+GJkEM8REM8MJYR5mYhIfzoQnjxqPdBI3oHGLMhFjQGQyow4gEFHAHSOLOFMUhGmgl/o+\nf9RCdYPa1be4dan6vM6p01W/7rr347X6y6+/dev+zN0REZHyEit1ABERKT4VdxGRMqTiLiJShlTc\nRUTKkIq7iEgZUnEXESlDH1rczazRzHaY2fq8sbiZPWVmr2a+9ju8MUVEpBBdmbn/Cji/09gtwFJ3\n/ySwNPNYREQiwrryISYzOxFY7O7DMo83AaPdfZuZDQSWu/vQwxlURES6rrqbzzvW3bdl7m8Hjn2/\nHzSzKcAUgLq6upEnnHBCN3dZHKlUilis9G81RCFHFDJEJUcUMkQlRxQyRCVHFDIAvPLKK//l7v0L\nepK7f+gNOBFYn/c42en7O7uynSFDhnipPf3006WO4O7RyBGFDO7RyBGFDO7RyBGFDO7RyBGFDO7u\nwCrvQo3Nv3X3n6R3M+0YMl93dHM7IiJyGHS3uD8GXJ25fzXwm+LEERGRYujKqZDzgWeBoWb2jplN\nBm4FzjWzV4FzMo9FRCQiPvQNVXef9D7fGlfkLCIiUiSlfxtYRESKTsVdRKQMdfc8dxER+QDbtv+G\nN16/jX37t1HXYyAf+/hNDBzw5dD2r+IuIlJk27b/ho0b/4FUai8A+/ZvZePGfwAIrcCrLSMiUmRv\nvH4bqdReXn+9gddfbwAgldrLG6/fFloGzdxFRIps3/701Vn+Z0+/Q46HQTN3EZEiq+sxsKDxw0HF\nXUSkyD728ZuIxXp2GIvFevKxj98UWga1ZUREiiz7pukf1y0jlWqhrscgnS0jIlIOBg74Mn36JAD4\n/OfvCH3/asuIiJQhFXcRkTKk4i4iUoYCFXczu9HM1pvZS2b27WKFEhGRYLpd3M1sGPC/gM8Cw4Hx\nZvaJYgUTEZHuCzJzPxl43t2b3b0NeAb4SnFiiYhIEJZee7UbTzQ7mfTyemcBe4GlpBdx/Vann5sC\nTAHo37//yKampkCBg9qzZw+9e/cuaYao5IhChqjkiEKGqOSIQoao5AiaYc2aNQCMGDEiUI4xY8as\ndveGgp5U6Ira+TdgMrAaWAHcAfzLB/38kCFDDt/y4F0UldXMo5AjChnco5EjChnco5EjChnco5Ej\naIbGxkZvbGwMnIP0xLmg+hzoDVV3n+fuI919FLATeCXI9kREpDgCfULVzI5x9x1mdgLpfvuZxYkl\nIiJBBL38wENm9hGgFfhbd08WIZOIiAQUqLi7+xeLFURERIpHn1AVESlDKu4FSCYSH/hYpJT0+pR8\nKu5d9NTUuayd0ZT7hUkmEqyd0cRTU+eWOJkIzPvurSy69e4Or89Ft97NvO/eWuJklWvx4sVs3ryZ\nzZs3M2vWLBYvXhzq/lXcuyCZSFBDLZ/oc0quwK+d0cQn+pxCDbWaIUlJJRMJUhZjS6/mXIFfdOvd\nbOnVTMpien2WwOLFi1m1alXusbuzatWqUAu8insX1MfjnDr7Ul7b/RKf6HMKe36S/vra7pc4dfal\n1MfjpY4oFaw+HmfCLdcxuLkXW3o18y9z57KlVzODm3sx4Zbr9PosgdWrVwNwdKonR6d6HjQeBhX3\nLsoW+Hwq7BIV2QKfT4W9dDxzWZePpHrzkVTvg8bDoOLeRdlWTL78HrxIKWVbMfnye/ASLjMD4JT2\n4zml/fiDxsOg4t4F+T3213a/RO/vnZJr0ajAS6nl99gHN/fi29Om5Vo0KvClMXLkyILGDwcV9y6o\nj8dppaVDjz3bg2+lRX/6SknVx+PEPNWhx57twcc8pddnCYwfP56GhgbITNTNjIaGBsaPHx9ahqCX\nH6gY594+jWQikftFyRZ4/eJIFEz+6S0HvT7Vcy+t8ePHs2PLHwGYef3Foe9fM/cCdP5F0S+ORIle\nn5JPxV1EpAypuIuIlKFAxd3M/s7MXjKz9WY238zqihVMRES6r9vF3cwGA9OABncfBlQBE4sVTERE\nui9oW6Ya6Glm1UAvYGvwSCIiEpQF+Tismd0I/BOwF/idu3/tED8zBZgC0L9//5FNTU2dfyRUUVhR\nPSo5opAhKjmikCEqOaKQISo5gmYY/Hx6/rzljFSgHGPGjFnt7g0FPanQFbWzN6AfsAzoD9QAjwJX\nfNBzhgwZEngV8KCisKK6ezRyRCGDezRyRCGDezRyRCGDezRyBM3w7p3r/N071wXOAazyAmt0kLbM\nOcCb7v6eu7cCDwOfC7A9EREpkiDF/W3gTDPrZemr4YwDNhQnloiIBNHt4u7uzwMPAi8AL2a2dVeR\ncomISACBri3j7jOBmUXKIiIiRaJPqIqIlCEVdxEpO52vYV+Ka9qXOoMu+SsiZeUX136fWKoHE+ZM\noz4eTy9mMn0uqdh+rm/8USgZ5l5xLT3ozYWDv4qZ8fbNz/DElofYzx6m3dcYSgbN3EWkbCQTCWKp\nHrTUjWLR9Lm5wt5SN4pYqkcos+dkIkEPerOn9Q12tu7E3Vm6Yyl7Wt+gB71Dm8GruItI2aiPx5kw\nZxq1+1bQUjeK+/9+LS11o6jdtyI3kw8jwwWD/4Z4r9Noa9/De/v+RKL5BeK9TuOCwX8T2nX2VdxF\npKxkC3y+sAp7VsxijOs/jmTLDpItOwAY138cMQuv5Kq4i0hZybZi8mVbNGFJeYql7y1lTSJ9A1j6\n3lJSHuwaM4VQcReRspHfY6/dt4Kv/fOpuRZNWAU+mUjwxJaHSDS/QHWsN/17HE+812kkml/giS0P\nqecuIlKo+nicVGx/hx57tgefiu0Pree+nz30rvkY/Wr6gcG4Y8bRu+Zj7GdPaO0hnQopImXl+sYf\nkUwkckU0W+DD7LlPu6+RZCLBk//3JwCcMPNsLkt8Oty+f2h7EhEJSeciGmZRjUoGFXcRkTIUZA3V\noWa2Nu+228y+XcxwIiLSPd3uubv7JuBUADOrArYAjxQpl4iIBFCstsw44HV331yk7YmISADFKu4T\ngflF2paIiARk6bVXA2zArBbYCpzi7u8e4vtTgCkA/fv3H9nU1BRof0FFYUX1qOSIQoao5IhChqjk\niEKGqOQImmHTbxYAMPTLEwPlGDNmzGp3byjoSYWuqN35BnwZ+F1XfnbIkCGBVwEPKgorqrtHI0cU\nMrhHI0cUMrhHI0cUMrhHI0fQDAt+eLMv+OHNgXMAq7zA2lyMtswk1JIREYmUQMXdzI4CzgUeLk4c\nEREphqALZP8P8JEiZRERkSLRJ1RFRMqQirtImSj1gsxRySBpKu4iZWDKz8cyff7FuWKaTCSYPv9i\npvx8bGgZGq88ncdvGNchw+M3jKPxytNDyxAlv7/7dt55eT3vvLyeOZMu5vd33x7q/lXcRY5wyUSC\nNmthZe9duQI/ff7FrOy9izZrCW2BipqWNhrW7csV+MdvGEfDun3UtLRV3Az+93ffzrqnfpt77KkU\n6576bagFXsVd5AhXH48zZ9JjnL6nLyt77+KLj5/Nyt67OH1PX+ZMeiy0BSouumMpq4bX0bBuH9s+\n93ka1u1j1fA6LrpjaUkuuVtKf1y6JH0n1jd96zweAhV3kTKQLfD5wirs+RkuumNph7FKLOyQnqkD\nxKqOIVZ1zEHjYVBxFykD2VZMvvwefFgZHr9hXIex/B58JbFYurRW9ziV6h6nHjQeBhV3kSNcfo/9\n9D19+beLnsm1aMIq8Pk99lXD6xj4H/+ea9FUYoH/zLjzCxo/HFTcRY5w9fE41V7bocee7cFXe21o\nPffW2uoOPfZsD761trriWjPnXDeV4edeCGZAesY+/NwLOee6qaFl0ALZImXgrm8uO2hR6LB77tfe\nu/KgDJXac4d0gf/zrhcA+OvvXB36/jVzFykTpV6QOSoZJE3FXUSkDKm4i4iUoaCX/K03swfNbKOZ\nbTCzs4oVTEREui/oG6o/A5a4+1czy+31KkImEREJqNvF3cz6AqOAawDcvQVoKU4sEREJotsLZJvZ\nqcBdwMvAcGA1cGNmAY/8n9MC2RHNEYUMUckRhQxRyRGFDFHJETTDm0vTlxv46Lhgb2+GukA20AC0\nAWdkHv8M+N8f9BwtkH1AFHJEIYN7NHJEIYN7NHJEIYN7NHIEzfDwbav94dtWB85ByAtkvwO84+7P\nZx4/CJwWYHsiIlIk3S7u7r4d+JOZDc0MjSPdohERkRILerbMt4D7M2fKvAF8PXgkEREJKlBxd/e1\npHvvIiISIfqEqohIGVJxFykTOztdM73z4zB0vm57pV3HPUpU3EXKwOif3csF9zyZK+g7EwkuuOdJ\nRv/s3tAy/OrWiSy5c3KuoCcTCZbcOZlf3ToxtAxR8swDG9n6apKtrya5feoynnlgY6j7V3EXOcLt\nTCTY59Vs394nV+AvuOdJtm/vwz6vDmUGn0wkqEvtZ2LLslyBX3LnZCa2LKMutb/iZvDPPLCR9Su2\n5h57Ctav2BpqgVdxFznC9YvHeeKq8xgwYDfbt/dhxE+eZfv2PgwYsJsnrjqPfiGtxHT+N+axoHYs\nE1uWUT/3o0xsWcaC2rGc/415FXdd95f+kC7sPZvfo2fzeweNh0HFXaQMZAt8vrAKe1a2wOerxMIO\n6Zk6QO8979B7zzsHjYdBxV2kDGRbMfnye/BhyLZi8uX34CuJZSrrcVue4bgtzxw0HgYVd5EjXH6P\nfcCA3az53lm5Fk1YBT6/x76gdizJaW/mWjSVWOBP+cKggsYPBxV3kSNcv3icOmvr0GPP9uDrrC20\nnvu+WI8OPfZsD35frEfFtWbOvvwkho0aBJZ+bDEYNmoQZ19+UmgZgl5+QEQiYPmNV7IzkcgV8myB\nD7Pnfs0tC0gmErlCni3wlVbYs86+/CQ2P1EPwOduHxv6/jVzFykTnQt5mIU9q3Mhr9TCHgUq7iIi\nZUjFXUSkDAXquZvZW8CfgXagzQtdBkpERA6LYryhOsbd/6sI2xERkSJRW0ZEpAxZeu3Vbj7Z7E1g\nF+m2zC/c/a5D/MwUYApA//79RzY1NXV7f8UQhRXVo5IjChmikiMKGaKSIwoZopIjaIZ+/2cOADu/\nMz1QjjFjxqwuuO1d6Ira+TdgcObrMcA6YNQH/fyQIUMCrwIeVBRWVHePRo4oZHCPRo4oZHCPRo4o\nZHCPRo6gGd664kp/64orA+cAVnmB9TlQW8bdt2S+7gAeAT4bZHsiIlIc3S7uZnaUmR2dvQ/8FbC+\nWMFERKT7gpwtcyzwiJllt/OAuy8pSioREQmk28Xd3d8Ahhcxi4iIFIlOhRQRKUMq7tItna8RHuai\nECLy4VTcpWDXPjiXqSsW5gr6zkSCqSsWcu2Dc0ucTCQ6ts2aRfPKlTSvXMmGU4axbdasUPev4i4F\n2ZlIsLe6hqf7npUr8FNXLOTpvmext7pGM3gR0oU9OX/BgYH2dpLzF4Ra4FXcpSD94nFuH3UZY3Y9\ny9N9z+LkdW/zdN+zGLPrWW4fdVlJriEuEjXJpkUAbK9P3zqPh0HFXQqWLfD5VNhF8rS3A/DWscZb\nx9pB42FQcZeCZVsx+fJ78CIVr6oKgCUjYywZGTtoPAwq7lKQ/B77mF3PsmH4CbkWjQq8SFr9pRMK\nGj8cVNylIP3icXq2tXbosWd78D3bWtWaEQEGzpxJ/aSJkO3IVFVRP2kiA2fODC1DMRbrkArT+NVp\n7EwkcoU8W+BV2EUOGDhzJr2WvAXAyT/4Zej718xduqVzIVdhF4kWFXcRkTKk4i4iUoYCF3czqzKz\nNWa2uBiBREQkuGLM3G8ENhRhOyIiUiSBiruZHQd8Cbi7OHFERKQYLL32ajefbPYg8CPgaOAmdx9/\niJ+ZAkwB6N+//8impqZu768YorCielRyRCFDVHJEIUNUckQhQ1RyBM3ws+0/A+DGATcGyjFmzJjV\n7t5Q0JMKXVE7ewPGA7dn7o8GFn/Yc4YMGRJ4FfCgorCiuns0ckQhg3s0ckQhg3s0ckQhg3s0cgTN\ncM0T1/g1T1wTOAewygus0UHaMp8HLjazt4AFwFgzuy/A9kREpEi6Xdzd/fvufpy7nwhMBJa5+xVF\nSyYiIt2m89xFRMpQUa4t4+7LgeXF2JaIiASnmbuISBlScS9AstO1yjs/riRRORZRySESNbrkbxc9\nNXUuNdRy6uxLqY/HSSYSrJ3RRCstnHv7tFLHC9W8795KymJMuOW63LFYdOvdxDzF5J/eElqOh34x\nAatpYewlv8zlWPbo1/HWWv7m+vDWqtyw8Qds3boAaAeqGDRoIief9I+h7T/r5k1vc9/WRCYFXDEo\nzo+HnhBqhhmPvsj85/9EuztVZkw643hmX/LpUDMAsHg6rP4VeDtYFYy8BsbPCTXC7Odms+rdVQAM\nv2c4E4ZMYMaZM0Lbv2buXZBMJKihlk/0OYW1M5pyhf0TfU6hhtqKmi0mEwlSFmNLr2YW3Xp3rrBv\n6dVMymKhHYtkIoHVtND3xPUse/TrucLe98T1WE1LaDnShf1+0oUdoJ2tW+9nw8YfhLL/rJs3vc2v\nM4WdTJpfb01w86a3Q8sw49EXue+5t2nPfDCy3Z37nnubGY++GFoGIF3YV81LF3ZIf101Lz0ektnP\nzWbhpgNLUaY8xcJNC5n93OzQMgT6hGqhhg4d6ps2bQptf4eyfPlyRo8eXfDz8gt61mu7X8rN5MPK\nUUxBjkW2oGcNbu6Vm8mHmSNb0LN2vTUsN5MPI8PSZUOAdpLJY+jRo5mePffkvldff0bB20smk9TX\n1xf8vGeTe3Dgd1zAZk5khw3Mfe+s+qMKzLCL+vq+BWd4/o0EDlS9vYfY7lZiew8sBn3GRwt/XXT3\nWLD538Gdq6t+x6dimzkx9u6B7/3lF0LJsPLdlQD8ON6P5pjxp5oaAGIWY91V6wrenpkV/AlVzdy7\nqD4e59TZl3YY625hP9LVx+NMuOW6DmPdLexBc4y9pOMKN90p7MGkC1hrax3t7TUh7rej8KZo7y8K\nGQAIccL6YY5va+WYtgP/yKU8Fdq+1XPvokPN3NfOaKrIAp+dudPrwNiiW+8OvcAfmLkfGFv26NdD\nLvBVQDvbtg4F4DPDn8qNjzztgYK3tnz5ckaeNrrg51389FraDzFeBTwy4pMFZxg9orDLmAB8vOm3\nuZZMhwxmLLz+rIK3l/5rqvDnMetLB1oy+awKvv6vBW1qbTf/orvunuGHLOQxC28+rZl7F+QX9td2\nv0Tv76W/5vfgK0V+S2Zwcy++PW0ag5t7dejBh5Uj25LZ9dYwRp66kl1vDevQgw/DoEETCxo/XK4Y\ndOh/zN5v/HCYdMbxBY0fNiOvKWz8MJgwZEJB44eDinsX1MfjtNLSocd+6uxLeW33S7TSUlEz9/p4\nnJinOvTYJ9xyHYObexHzVGjHoj4ex1trO/TYx17yS3a9NQxvrQ0tx8kn/SODBn2NvGXuGTToa6Gf\nLfPjoSdw9aA4VbkUcHXIZ8vMvuTTXHHmCVRZ+lhUmXHFmSeEf7bM+DnQMDk9U4f014bJoZ4tM+PM\nGVw29LLcTD1mMS4belmoZ8t0+6qQ3bkd6VeF3Pnf//2Bj8PKUSzlcCyKlSPo/4/GxkZvbGwMtI1i\n5CiGKGRwj0aOKGRwD/+qkBWn82ywkmbsnUXlWEQlh0jUqLiLiJShbhd3M6szs/80s3Vm9pKZzSpm\nMBER6b4gp0LuB8a6+x4zqwH+YGZPuPtzRcomIiLd1O3inmnyZz+OV5O5RefTAyIiFSxQz93Mqsxs\nLbADeMrdny9OLBERCaIo15Yxs3rgEeBb7r6+0/emAFMA+vfvP7KpqSnw/oKIworqUckRhQxRyRE0\nw5o1awAYMWJESXMUQxQyRCVHFDIAjBkzpuBryxTtHHbgB8BNH/QzR/p57sUUhRxRyOAejRw6zz1a\nGdyjkSMKGdxDPs/dzPpnZuyYWU/gXGBjd7cnIiLFE+RsmYHAr82sinTvvsndFxcnloiIBBHkbJk/\nAsEajCIicljoE6oiImVIxb0AWoz5AB0LORS9Lg4o9bHQYh1dNOXnY2mzFuZMeiy3GPP0+RdT7bXc\n9c1lpY4XqsYrT6empY2L7liaOxaP3zCO1tpqrr13ZahZEo++SvPz29MfnzPodcYA4pcUtjhFUIsX\nL2bz5s0AzJo1i5EjRzJ+/PhQM0TBL679PrFUDybMmXZg4fTpc0nF9nN9449KHS9Uc6+4FkvBlT+/\nLXcs7v3mTXgMpt3XGEoGzdy7IJlI0GYtrOy9i+nzL84V9pW9d9Fm4S3GHAXJRIKaljYa1u3j8RvG\n5Qp7w7p91LS0hXosEo++SvNz2w98Ltqh+bntJB59NbQMixcvZtWqVbnH7s6qVatYvLiyzi1IJhLE\nUj1oqRvFoulzc4W9pW4UsVSPivsdsRS0tO/g3m/elCvsLe07sFR4M3jN3LugPh5nzqTHcgX9i4+f\nDb3h9D19czP5SlEfj3PRHUtzBX3b5z5PA7BqeF1uJh+W5ue3A7Bj79v0rD6ao2v6pcef207bu3sL\n2tbgZIwdm/5YcIZjN6f4kp/GszWv0Eo7f46l97t69eqKmr3Xx+NMmDMtV9Dv//u1UDeK2n0rcjP5\nSlEfj3Plz2/LFfR5N1wFQG3VMbmZfBg0c++ibIHPV2mFPStb4POFXdiB3Ix9f6qZtlRLuPvulKFP\nqidHeY8DwxFapDks2QKfr9IKe1a2wOcLs7CDZu5dlm3FkPdJ5OnzL67IAp9rxeSNPX7DuPALvAEO\nr+5+AYCxAy/PjR9z/WcK2tTLy5fzydGFPQfgjlmPHLKQm9khfrq8ZVsx1I3KjS2aPrciC3y2FZPv\n3m/epJl71OT32E/f05d/u+gZTt/Tt0MPvlLk99hXDa9j4H/8O6uG13XowYel1xkDCho/HEaOHFnQ\neLnK77HX7lvB1/75VGr3rejQg68U+T322qpjmHzHPdRWHdOhBx8GFfcuqI/HqfbaDj32OZMe4/Q9\nfan28BZjjoL6eJzW2uoOPfaL7ljKquF1tNZWh3os4pd8kl5n5hVyg15nhnu2zPjx42loaMjN1M2M\nhoaGiuq3Q/p1kYrt79BjnzBnGrX7VpCK7a+43xGPdeyxX/nz26itOgaPhbcUpNoyXXTXN5eRTCRy\n/2OyBb6SXrRZ19678qBjUZKeO+kC32NdXwCOm/nF0PcP6QJfacX8UK5v/NFBr4tKbMlA+nTHzsci\n7J67Zu4F0GLMB+hYyKHodXFAqY+FiruISBlScRcRKUNBrud+vJk9bWYvm9lLZnZjMYOJiEj3BXlD\ntQ34jru/YGZHA6vN7Cl3f7lI2UREpJu6PXN3923u/kLm/p+BDcDgYgUTEZHuK9YC2ScCK4Bh7r67\n0/e0QHZEc0QhQzFybPrNAgCGfnliyTIUSxRyRCFDVHJEIQOUaIFs0h/IXw185cN+VgtkHxCFHFHI\n4B48x4If3uwLfnhzSTMUSxRyRCGDezRyRCGDe8gLZAOYWQ3wEHC/uz8cZFsiIlI8Qc6WMWAesMHd\n5xQvkoiIBBVk5v554EpgrJmtzdwuLFIuEREJoNunQrr7H0hfdFVERCJGn1AVESlDKu4iZaLzdcIr\n6RrqcjBd8rcAs5+bzaJXFpHyFDGLMWHIBGacOSP0HNtmzSLZtAja26GqivpLJzBw5sxQMzzzwEZe\n+sNWPAUWg1O+MIizLz8p1AwAv7/7dt55eT0AcyZdzGfGnc85100NPUep/erWidSl9nP+N+ZRH4+T\nTCRYcudk9sV6cM0tC0odT0pAM/cumv3cbBZuWkjKUwCkPMXCTQuZ/dzsUHNsmzWL5PwF6cIO0N5O\ncv4Cts2aFVqGZx7YyPoV6cIO4ClYv2IrzzywMbQMkC7s6576be6xp1Kse+q3/P7u20PNUWrJRIK6\n1H4mtixjyZ2Tc4V9Yssy6lL7NYOvUJq5d9GiVxYBMHLvPnZUV/GnmhoAFm5ayOvJ1wveXjKZ5NdL\nfl3w85pjK+HyKs5fneLEd50Bycz25i+g5bXCcvRLJtk8r7HgDDWvJhnh8M7gs9nT+zj29uoPpAt8\nYltzwdtLJlPsXP1Cwc97Z0NvantPoLV5OU4LpHYB8MelSypq9l4fj3P+N+axIFPQmftRJgILasfm\nZvJSeTRz76LsjL1fqp1eqRKubF/CXedEIQNA5tIZVlWP2YGPiHsqVapEJZMt8PlU2CubZu5dFLMY\nKU9x+e49AFw78Njc+C/P/2XB21u+fDmjR48u+HkbvjPsQEsmX1UVf3nvPQVt683lyxnejQz/OnVZ\nriWTz2Lw1985reDtpY9F4c+bM+mHhyzkFqu8OUuuFZM3tuTOySrwFazyfgu6acKQCQWNHy71lx56\nf+83fjic8oVBBY0fLp8Zd35B4+Uqv8e+oHYsyWlvsqB2bIcevFQeFfcumnHmDC4belnuccxiXDb0\nstDPlhk4cyb1kyZCVVV6oKqK+kkTQz1b5uzLT2LYqEFY5tVjMRg2KvyzZc65birDz70wN1O3WIzh\n515YUf12SLdk9sV6dOixn/+NeSyoHcu+WA/N3CuU2jIFmHHmDNjwLADrrvrXkuUYOHNm6Kc+dnb2\n5SeV5NTHzs65bmrFFfNDueaWBSQTiVwhzxZ4FfbKpZm7SJnoXMhV2CubiruISBlScRcRKUNBF+to\nNLMdZra+WIFERCS4oDP3XwGVdd6ZiMgRIFBxd/cVgE6iFRGJGHMP9llyMzsRWOzuw97n+1OAKQD9\n+/cf2dTUFGh/QQVdzfzUNf8AwNoR/1TSHMUQhQxRyRGFDFHJEYUMUckRhQwAY8aMWe3uDQU9qdAV\ntTvfgBOB9V352SFDhhyepcELEHg188YL07dS5yiCKGRwj0aOKGRwj0aOKGRwj0aOKGRwdwdWeYG1\nWWfLiIiUIRV3EZEyFPRUyPnAs8BQM3vHzCYXJ5aIiAQR6Noy7j6pWEFERKR41JYRESlDKu7SLTs7\nXSO88+PalKy9AAAHzUlEQVRKomMhUXTEXPL30TVb+OmTm9ia3Mug+p5897yhXDJicLghFk+HzX9I\n358Vh5HXwPg54WaIgGsfnMve6hpuH3UZ/eJxdiYSTF2xkJ5trTR+dVqp44Vq9M/uZZ9X88RV5+WO\nxQX3PEmdtbH8xitLHU8q2BExc390zRa+//CLbEnuxYEtyb18/+EXeXTNlvBCLJ4Oq/LWqPT29OPF\n08PLEAE7Ewn2VtfwdN+zmLpiYa6wP933LPZW11TUrHVnIsE+r2b79j5ccM+TucK+fXsf9nl1RR0L\niZ4jYub+0yc3sbe147qhe1vb+d6Df2T+f75d0LaSyb3csenZwkNsHgQ+gx/U3MNR7OfE2Lvp8dW/\nqqjZe794nNtHXZYr6Cevexv6nsWYXc/mZvKVol88zhNXnZcr6CN+8izQhwEDdudm8iKlckTM3Lcm\n9x5yvKU9xFXuM5dp2JwawHbvlzd+iMWqy1y2wOertMKelS3w+VTYJQqOiJn7oPqebDlEgR9c35OF\n159V0LaWL1/O6NGFPQeAWV86dCG3qsK3dYTLtmLoe+A4Tl2xsCILfLYVA31yYxfc86QKvJTcETFz\n/+55Q+lZ07GI9qyp4rvnDQ0vxMhrChsvU/k99jG7nmXD8BMYs+vZDj34SpHfYx8wYDdrvncWAwbs\n7tCDFymVI6K4XzJiMD/6yqcZXN8TIz1j/9FXPh3u2TLj50DD5AMzdatKP66gfjuk2xA921o79Nhv\nH3UZY3Y9S8+21oqarfaLx6mztg499ieuOo8BA3ZTZ20VdSwkeo6ItgykC3zopz52Nn5OxRXzQ2n8\n6jR2JhK54pUt8JVYzJbfeOVBx0ItGYmCI2LmLtHTuXhVcjHTsZAoUnEXESlDKu4iImUo6CV/zzez\nTWb2mpndUqxQIiISTLeLu5lVAf8PuAD4FDDJzD5VrGAiItJ9QWbunwVec/c33L0FWAB8uTixREQk\niCCnQg4G/pT3+B3gjM4/ZGZTgCmZh/vNbH2AfRbDXwD/VeIMEI0cUcgA0cgRhQwQjRxRyADRyBGF\nDAAFf2LzsJ/n7u53AXcBmNkqd2843Pv8IFHIEJUcUcgQlRxRyBCVHFHIEJUcUciQzVHoc4K0ZbYA\nx+c9Pi4zJiIiJRakuK8EPmlmHzWzWmAi8FhxYomISBDdbsu4e5uZfRN4EqgCGt39pQ952l3d3V8R\nRSEDRCNHFDJANHJEIQNEI0cUMkA0ckQhA3Qjh3nmOuUiIlI+9AlVEZEypOIuIlKGQinuUbhMgZk1\nmtmOUp5nb2bHm9nTZvaymb1kZjeWKEedmf2nma3L5JhVihyZLFVmtsbMFpcww1tm9qKZre3OKWdF\nylBvZg+a2UYz22Bm3VguLHCGoZljkL3tNrNvlyDH32Vel+vNbL6Z1YWdIZPjxkyGl8I8DoeqVWYW\nN7OnzOzVzNd+H7QNANz9sN5Iv9n6OvAxoBZYB3zqcO/3EDlGAacB68Ped16GgcBpmftHA6+U6FgY\n0DtzvwZ4HjizRMdkOvAAsLiE/1/eAv6iVPvPZPg1cF3mfi1QX+I8VcB24C9D3u9g4E2gZ+ZxE3BN\nCf77hwHrgV6kTzz5PfCJkPZ9UK0CfgLckrl/C/DjD9tOGDP3SFymwN1XACVd98zdt7n7C5n7fwY2\nkH4xh53D3X1P5mFN5hb6O+tmdhzwJeDusPcdJWbWl/Qv9DwAd29x92RpUzEOeN3dN5dg39VATzOr\nJl1ct5Ygw8nA8+7e7O5twDPAV8LY8fvUqi+TngCQ+XrJh20njOJ+qMsUlHhJpdIzsxOBEaRnzaXY\nf5WZrQV2AE+5eyly/AvwPSBVgn3nc+D3ZrY6c7mMsH0UeA/4ZaZFdbeZHVWCHPkmAvPD3qm7bwFu\nA94GtgG73P13YecgPWv/opl9xMx6ARfS8UObYTvW3bdl7m8Hjv2wJ+gN1RIws97AQ8C33X13KTK4\ne7u7n0r6k8WfNbNhYe7fzMYDO9x9dZj7fR9fyByLC4C/NbNRIe+/mvSf4Xe4+wjgf0j/6V0SmQ8l\nXgwsKsG++5GepX4UGAQcZWZXhJ3D3TcAPwZ+BywB1gLtYec4FE/3Zj70L+0wirsuU5DHzGpIF/b7\n3f3hUufJ/Pn/NHB+yLv+PHCxmb1FulU31szuCzkDkJst4u47gEdItxLD9A7wTt5fTw+SLvalcgHw\ngru/W4J9nwO86e7vuXsr8DDwuRLkwN3nuftIdx8F7CT9HlmpvGtmAwEyX3d82BPCKO66TEGGmRnp\nvuoGdy/ZSttm1t/M6jP3ewLnAhvDzODu33f349z9RNKviWXuHvoMzcyOMrOjs/eBvyL9J3lo3H07\n8Cczy175bxzwcpgZOplECVoyGW8DZ5pZr8zvyzjS702FzsyOyXw9gXS//YFS5Mh4DLg6c/9q4Dcf\n9oQwrgrZncsUFJ2ZzQdGA39hZu8AM919XsgxPg9cCbyY6XcD/L27/zbkHAOBX2cWXIkBTe5eslMR\nS+xY4JF0HaEaeMDdl5Qgx7eA+zMToDeAr5cgQ/YfuHOB60uxf3d/3sweBF4A2oA1lO4SAA+Z2UeA\nVuBvw3qT+1C1CrgVaDKzycBm4NIP3U7m1BoRESkjekNVRKQMqbiLiJQhFXcRkTKk4i4iUoZU3EVE\nypCKu4hIGVJxFxEpQ/8fy802s1NK/uIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1e3146fcd68>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from collections import deque\n",
    "import random\n",
    "import matplotlib.pyplot as plt\n",
    "class Q_Network:\n",
    "    N_Mesh=10\n",
    "    State_Number=N_Mesh*N_Mesh;\n",
    "    Action_Number=4;\n",
    "    Action_List=np.identity(Action_Number)\n",
    "    State_List=np.identity(State_Number)\n",
    "    Epsilon=0\n",
    "    Epsilon_Begin=1\n",
    "    Epsilon_Final=0.01\n",
    "    decay_rate=0.00001\n",
    "    Step=0\n",
    "    Batch_Number=20\n",
    "    Observe=1000#The number of steps of observation before the beginning of the training\n",
    "    Store_Memory = deque()\n",
    "    Cost_History=[]\n",
    "    Memory_Size = 5000\n",
    "    Obstacle=[20,3,5,16,21,23,27,29,31,33,34,56,57,66,67,78,81,86]\n",
    "    Obstacle=[]\n",
    "    Play_Done=None\n",
    "    def __init__(self,Learning_Rate=0.001,Gamma=0.9,Memory_Size=5000):\n",
    "        self.Learning_Rate=Learning_Rate\n",
    "        self.Gamma=Gamma\n",
    "        self.Memory_Size=Memory_Size\n",
    "        self.Draw_Obstacle()\n",
    "        self.New_Network()\n",
    "        self.session=tf.InteractiveSession()\n",
    "        self.session.run(tf.initialize_all_variables())\n",
    "     #***************************************************************\n",
    "    def New_Network(self):\n",
    "         #Construct a neural network\n",
    "        self.State_Input=tf.placeholder(tf.float32,[None,self.State_Number])#Input_Number=100, M=Batch*100\n",
    "        self.Action_Input=tf.placeholder(tf.float32,[None,self.Action_Number])#Input_Number=4, M=Batch*4\n",
    "        self.Q_Target_OI=tf.placeholder(dtype=tf.float32,shape=[None])\n",
    "        #Layer1\n",
    "        Layer1_Number=20\n",
    "        w1=tf.Variable(tf.random_normal([self.State_Number,Layer1_Number]))\n",
    "        b1=tf.Variable(tf.zeros([1,Layer1_Number])+0.1)\n",
    "        l1=tf.nn.relu(tf.matmul(self.State_Input,w1)+b1) #b1 use the propagation mechanism\n",
    "#         #Layer2\n",
    "#         Layer2_Number=10\n",
    "#         w2=tf.Variable(tf.random_normal([Layer1_Number,Layer2_Number]))\n",
    "#         b2=tf.Variable(tf.zeros([1,Layer2_Number]))\n",
    "#         l2=tf.nn.relu(tf.matmul(l1,w2)+b2)\n",
    "        #Layer3\n",
    "        Layer3_Number=self.Action_Number\n",
    "        w3=tf.Variable(tf.random_normal([Layer1_Number,Layer3_Number]))\n",
    "        b3=tf.Variable(tf.zeros([1,Layer3_Number]))\n",
    "        self.l3=tf.matmul(l1,w3)+b3#At this time, not yet use the active action, l3 is the output matrice, M=Batch*4\n",
    "\n",
    "        self.Q_Value=tf.reduce_sum(tf.multiply(self.l3,self.Action_Input),reduction_indices=1)\n",
    "        self.Loss=tf.reduce_mean(tf.square(self.Q_Value-self.Q_Target_OI))\n",
    "        self.Optimizer=tf.train.GradientDescentOptimizer(self.Learning_Rate).minimize(self.Loss)\n",
    "        #Network's Output is like this form [19,20,297,30], argmax is 2, then choose Action2  \n",
    "        self.Predict=tf.argmax(self.l3,1)\n",
    "     #***************************************************************\n",
    "    def Select_Action(self,State_Index):\n",
    "        Current_State=self.State_List[State_Index:State_Index+1] #By example, Action_Index is 2, Current_State=[0,0,1,0， ，]\n",
    "        if np.random.uniform()<self.Epsilon:\n",
    "            Choose_Action_Index=np.random.randint(0,self.Action_Number)\n",
    "        else:\n",
    "            Action_QValue_Output=self.session.run(self.l3,feed_dict={self.State_Input:Current_State})\n",
    "            Choose_Action_Index=np.argmax(Action_QValue_Output)\n",
    "        #The first inegality means the train has beginned\n",
    "        #The second inegality means Epsilon has not yet decreased to final value\n",
    "        self.Epsilon=self.Epsilon_Final+(self.Epsilon_Begin-self.Epsilon_Final)*np.exp(-self.decay_rate*self.Step)\n",
    "#         if(self.Step%1000==0):\n",
    "#             print(\"epsilon \",self.Epsilon)\n",
    "        return Choose_Action_Index\n",
    "     #***************************************************************\n",
    "    def Next_State(self,Action_Index,State_Index):\n",
    "        State=State_Index+1\n",
    "        done=False\n",
    "        #Execute left motion\n",
    "        if(Action_Index==0):\n",
    "            if(State%self.N_Mesh==1):\n",
    "                State=State\n",
    "                R=-1000\n",
    "            elif((State+1) in self.Obstacle):#When encounter obstables, we stay \n",
    "                State=State\n",
    "                R=-1000\n",
    "            else:\n",
    "                State=State-1\n",
    "                R=0\n",
    "        elif(Action_Index==1):\n",
    "            if(State==self.State_Number-1):\n",
    "                State=State+1\n",
    "                done=True\n",
    "                R=100\n",
    "            elif(State%self.N_Mesh==0):\n",
    "                State=State\n",
    "                R=-1000\n",
    "            elif((State+1) in self.Obstacle):\n",
    "                State=State\n",
    "                R=-1000\n",
    "            else:\n",
    "                State=State+1\n",
    "                R=0.02\n",
    "        elif(Action_Index==2):\n",
    "            if(State==self.State_Number-self.N_Mesh):\n",
    "                State=State+self.N_Mesh\n",
    "                done=True\n",
    "                R=100\n",
    "            elif(self.State_Number+1-self.N_Mesh<=State<=self.State_Number-1):\n",
    "                State=State\n",
    "                R=-1000\n",
    "            elif((State+self.N_Mesh) in self.Obstacle):\n",
    "                State=State+self.N_Mesh\n",
    "                R=-1000\n",
    "            else:\n",
    "                State=State+self.N_Mesh\n",
    "                R=0.02\n",
    "        else:\n",
    "            if(0<=State<=self.N_Mesh):\n",
    "                State=State\n",
    "                R=-100\n",
    "            elif((State-self.N_Mesh) in self.Obstacle):\n",
    "                State=State-self.N_Mesh\n",
    "                R=-100\n",
    "            else:\n",
    "                R=0\n",
    "                State=State-self.N_Mesh\n",
    "        return State-1,R,done\n",
    "    \n",
    "    def Save_Memory(self,CURRENT_STATET,CHOOSE_ACTION,NEXT_STATE,REWARD,DONE):\n",
    "        Memory_Current_State=self.State_List[CURRENT_STATET:CURRENT_STATET+1]\n",
    "        Memory_Choose_Action=self.Action_List[CHOOSE_ACTION:CHOOSE_ACTION+1]\n",
    "        Memory_Next_State=self.State_List[NEXT_STATE:NEXT_STATE+1]\n",
    "        self.Store_Memory.append((Memory_Current_State,Memory_Choose_Action,Memory_Next_State,REWARD,DONE))\n",
    "        if len(self.Store_Memory)>self.Memory_Size:\n",
    "            self.Store_Memory.popleft()\n",
    "    \n",
    "     #***************************************************************\n",
    "    def Experience_Replay(self):\n",
    "        Batch=self.Batch_Number\n",
    "        MiniBatch=random.sample(self.Store_Memory,Batch)\n",
    "        Batch_Current_State = None\n",
    "        Batch_Execute_Action = None\n",
    "        Batch_Reward = None\n",
    "        Batch_Next_State = None\n",
    "        Batch_Done = None\n",
    "        \n",
    "        for Index in range(Batch):\n",
    "            if Batch_Current_State is None:\n",
    "                Batch_Current_State=MiniBatch[Index][0]\n",
    "            elif Batch_Current_State is not None:\n",
    "                Batch_Current_State=np.vstack((Batch_Current_State, MiniBatch[Index][0]))\n",
    "            #---------------------------------------------------------------------------------------   \n",
    "            #print(\"BEA\",MiniBatch[Index][1].shape)\n",
    "            if Batch_Execute_Action is None:\n",
    "                Batch_Execute_Action=MiniBatch[Index][1]\n",
    "            elif Batch_Execute_Action is not None:\n",
    "                Batch_Execute_Action=np.vstack((Batch_Execute_Action,MiniBatch[Index][1]))\n",
    "           #---------------------------------------------------------------------------------------           \n",
    "            if Batch_Reward is None:\n",
    "                Batch_Reward=MiniBatch[Index][3]\n",
    "            elif Batch_Reward is not None:\n",
    "                Batch_Reward=np.vstack((Batch_Reward,MiniBatch[Index][3]))\n",
    "            #---------------------------------------------------------------------------------------  \n",
    "            if Batch_Next_State is None:\n",
    "                Batch_Next_State=MiniBatch[Index][2]\n",
    "            elif Batch_Next_State is not None:\n",
    "                Batch_Next_State=np.vstack((Batch_Next_State,MiniBatch[Index][2]))\n",
    "            #---------------------------------------------------------------------------------------      \n",
    "            if Batch_Done is None:\n",
    "                Batch_Done=MiniBatch[Index][4]\n",
    "            elif Batch_Done is not None:\n",
    "                Batch_Done=np.vstack((Batch_Done,MiniBatch[Index][4]))\n",
    "            #Calculate the Q Value of the next State \n",
    "        Q_Next=self.session.run(self.l3,feed_dict={self.State_Input:Batch_Next_State})\n",
    "        Q_Target=[]\n",
    "        for i in range(Batch):\n",
    "            Each_Reward=Batch_Reward[i][0]#This is a 2D array because of the vstack\n",
    "            #Calculate the Target-Q-Value of each element in the Batch\n",
    "            Each_QValue=Each_Reward+self.Gamma*np.max(Q_Next[i]) #The network ouput has its own []\n",
    "            if Each_Reward<0:\n",
    "                Q_Target.append(Each_Reward)\n",
    "            else:\n",
    "                Q_Target.append(Each_QValue)\n",
    "        #print(self.session.run(self.Q_Value,feed_dict={self.Q_Target_OI:Q_Target}))\n",
    "        _,Cost,Rew=self.session.run([self.Q_Value,self.Loss,self.Optimizer],feed_dict={self.State_Input:Batch_Current_State,\n",
    "                                                                                        self.Action_Input:Batch_Execute_Action,\n",
    "                                                                                        self.Q_Target_OI: Q_Target})\n",
    "        self.Cost_History.append(Cost)\n",
    "        if self.Step%5000==0:\n",
    "            print(self.Step)  \n",
    "            print(\"Loss:\", Cost)  \n",
    "            print(\"Epsilon:\", self.Epsilon)   \n",
    "    #**************************************************************\n",
    "    def Train(self):\n",
    "        Train_Current_State=np.random.randint(0,self.State_Number-1)\n",
    "        self.Epsilon = self.Epsilon_Begin\n",
    "        while True:\n",
    "            Train_Action=self.Select_Action(Train_Current_State)\n",
    "            #print(\"TA\",Train_Action)\n",
    "            Train_Next_State,Train_Reward,Train_Done=self.Next_State(Train_Action,Train_Current_State)\n",
    "            self.Save_Memory(Train_Current_State,Train_Action,Train_Next_State,Train_Reward,Train_Done)\n",
    "            if self.Step>self.Observe:\n",
    "                self.Experience_Replay()\n",
    "            if self.Step>500000:\n",
    "                 break;\n",
    "            if Train_Done:\n",
    "                Train_Current_State=np.random.randint(0,self.State_Number-1)\n",
    "            else:\n",
    "                Train_Current_State=Train_Next_State\n",
    "                self.Step+=1          \n",
    "    #***************************************************************\n",
    "    def Play(self):\n",
    "        self.Train()\n",
    "        Start_Room_Index=0;\n",
    "        Play_Current_State_Index=0\n",
    "        Play_Step=0;\n",
    "        print(\"****************** Agent is in the State\",Start_Room_Index,\"**********************\")\n",
    "        while(Play_Current_State_Index!=self.State_Number-1):\n",
    "            Play_Current_State=self.State_List[Play_Current_State_Index:Play_Current_State_Index+1]\n",
    "            Play_Action=(self.session.run(self.Predict,feed_dict={self.State_Input:Play_Current_State}))\n",
    "            if Play_Action==0:\n",
    "                Play_Current_State_Index=Play_Current_State_Index-1\n",
    "            elif Play_Action==1:\n",
    "                Play_Current_State_Index=Play_Current_State_Index+1\n",
    "            elif Play_Action==2:\n",
    "                Play_Current_State_Index=Play_Current_State_Index+self.N_Mesh\n",
    "            else:\n",
    "                Play_Current_State_Index=Play_Current_State_Index-self.N_Mesh\n",
    "            Play_Step+=1\n",
    "            print(\"Step is \",Play_Step,\"****************** Agent is in the State\", Play_Current_State_Index,\"**********************\")\n",
    "            if(Play_Step>30):\n",
    "                break\n",
    "        if Play_Current_State_Index==self.State_Number-1:\n",
    "            self.Graph()\n",
    "    #***************************************************************        \n",
    "    def Caculate_coordinate(self,State):\n",
    "        X_Axis=0.5+(State-1)%self.N_Mesh\n",
    "        Y_Axis=0.5+((State-1)//self.N_Mesh)\n",
    "        return X_Axis,Y_Axis \n",
    "    #***************************************************************  \n",
    "    def Graph(self): \n",
    "        Graph_Step=0\n",
    "        fig=plt.figure()\n",
    "        ax=fig.gca()\n",
    "        ax.set(xlim=[0, self.N_Mesh], ylim=[0, self.N_Mesh])\n",
    "        ax.set_xticks(np.arange(0,(self.N_Mesh+1)))\n",
    "        ax.set_yticks(np.arange(0,(self.N_Mesh+1)))\n",
    "        plt.grid()\n",
    "        Graph_Start_Index=0\n",
    "        Graph_Current_Index=Graph_Start_Index\n",
    "        while(Graph_Current_Index!=self.State_Number-1):\n",
    "            Graph_Current_State=self.State_List[Graph_Current_Index:Graph_Current_Index+1]\n",
    "            Graph_Action=(self.session.run(self.Predict,feed_dict={self.State_Input:Graph_Current_State}))\n",
    "            if(Graph_Action==0):\n",
    "                Graph_Next_Index=Graph_Current_Index-1\n",
    "            elif(Graph_Action==1):\n",
    "                Graph_Next_Index=Graph_Current_Index+1\n",
    "            elif(Graph_Action==2):\n",
    "                 Graph_Next_Index=Graph_Current_Index+self.N_Mesh\n",
    "            else:\n",
    "                 Graph_Next_Index=Graph_Current_Index-self.N_Mesh\n",
    "            Graph_Step+=1\n",
    "            X_State,Y_State=self.Caculate_coordinate(Graph_Current_Index+1)\n",
    "            X_Next_State,Y_Next_State=self.Caculate_coordinate(Graph_Next_Index+1)\n",
    "            if(Graph_Step==0):\n",
    "                plt.scatter(X_State,Y_State)  \n",
    "            else:\n",
    "                plt.scatter(X_State,Y_State) \n",
    "                plt.scatter(X_Next_State,Y_Next_State)\n",
    "                plt.plot([X_State,X_Next_State],[Y_State,Y_Next_State])\n",
    "            for J in np.arange(len(self.Obstacle)):\n",
    "                plt.scatter((self.Caculate_coordinate(self.Obstacle[J]+1))[0],(self.Caculate_coordinate(self.Obstacle[J]+1))[1],marker=\"x\")\n",
    "            Graph_Current_Index=Graph_Next_Index\n",
    "    plt.show()\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    q_network = Q_Network()\n",
    "    q_network.Play()\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "np.random.randint(10000,size=1000)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
